rm(list = ls())

### R package
# # only need to install once
# devtools::install_github("cloudyr/pyMTurkR", dependencies=TRUE)
# devtools::install_github("Luwei-Ying/validateIt", dependencies=TRUE)
# load R package
library(validateIt)

### load the crowdsourced results

# random 4 word set intrusion (R4WSI) with random documents
r4wsi_first <- vector() # first trial
r4wsi_second <- vector() # second trial

load("topicrecords/R4WSIstm10k1it.Rdata")
result <- read.csv("topicresults/R4WSIstm10k1itresults1.csv", stringsAsFactors = FALSE)
r4wsi_first[1] <- as.numeric(evalResults(results = result, key = record, type = "R4WSI0")[[3]][1])
result <- read.csv("topicresults/R4WSIstm10k1itresults2.csv", stringsAsFactors = FALSE)
r4wsi_second[1] <- as.numeric(evalResults(results = result, key = record, type = "R4WSI0")[[3]][1])

load("topicrecords/R4WSIstm10k.Rdata")
result <- read.csv("topicresults/R4WSIstm10kresults1.csv", stringsAsFactors = FALSE)
r4wsi_first[2] <- as.numeric(evalResults(results = result, key = record, type = "R4WSI0")[[3]][1])
result <- read.csv("topicresults/R4WSIstm10kresults2.csv", stringsAsFactors = FALSE)
r4wsi_second[2] <- as.numeric(evalResults(results = result, key = record, type = "R4WSI0")[[3]][1])

load("topicrecords/R4WSIstm50k.Rdata")
result <- read.csv("topicresults/R4WSIstm50kresults1.csv", stringsAsFactors = FALSE)
r4wsi_first[3] <- as.numeric(evalResults(results = result, key = record, type = "R4WSI0")[[3]][1])
result <- read.csv("topicresults/R4WSIstm50kresults2.csv", stringsAsFactors = FALSE)
r4wsi_second[3] <- as.numeric(evalResults(results = result, key = record, type = "R4WSI0")[[3]][1])

load("topicrecords/R4WSIstm100k.Rdata")
result <- read.csv("topicresults/R4WSIstm100kresults1.csv", stringsAsFactors = FALSE)
r4wsi_first[4] <- as.numeric(evalResults(results = result, key = record, type = "R4WSI0")[[3]][1])
result <- read.csv("topicresults/R4WSIstm100kresults2.csv", stringsAsFactors = FALSE)
r4wsi_second[4] <- as.numeric(evalResults(results = result, key = record, type = "R4WSI0")[[3]][1])

load("topicrecords/R4WSIstm500k.Rdata")
result <- read.csv("topicresults/R4WSIstm500kresults1.csv", stringsAsFactors = FALSE)
r4wsi_first[5] <- as.numeric(evalResults(results = result, key = record, type = "R4WSI0")[[3]][1])
result <- read.csv("topicresults/R4WSIstm500kresults2.csv", stringsAsFactors = FALSE)
r4wsi_second[5] <- as.numeric(evalResults(results = result, key = record, type = "R4WSI0")[[3]][1])

r4wsi_pool <- (r4wsi_first + r4wsi_second)/2

# # results so far should be as follows:
# r4wsi_first <- c(0.266, 0.856, 0.868, 0.958, 0.580786)
# r4wsi_second <- c(0.290, 0.654, 0.702, 0.754, 0.578)
# r4wsi_pool <- c(0.278, 0.755, 0.785, 0.856, 0.579393)

# random 4 word set intrusion (R4WSI) with representative documents
# These tasks were posted with a different system that have been depreciated.
# Therefore, the structures of these results are different from those in the article.
r4wsi2_first <- vector() # first trial
r4wsi2_second <- vector() # first trial

result <- read.csv("topicresults/V2stm1itR1.csv")
r4wsi2_first[1] <- sum(result$CorrectDummy)/nrow(result)
result <- read.csv("topicresults/V2stm1itR2.csv")
r4wsi2_second[1] <- sum(result$CorrectDummy)/nrow(result)

result <- read.csv("topicresults/V2stm10kR1.csv")
r4wsi2_first[2] <- sum(result$CorrectDummy)/nrow(result)
result <- read.csv("topicresults/V2stm10kR2.csv")
r4wsi2_second[2] <- sum(result$CorrectDummy)/nrow(result)

result <- read.csv("topicresults/V2stm20kcR1.csv")
r4wsi2_first[3] <- sum(result$CorrectDummy)/nrow(result)
result <- read.csv("topicresults/V2stm20kcR2.csv")
r4wsi2_second[3] <- sum(result$CorrectDummy)/nrow(result)

result <- read.csv("topicresults/V2stm100kR1.csv")
r4wsi2_first[4] <- sum(result$CorrectDummy)/nrow(result)
result <- read.csv("topicresults/V2stm100kR2.csv")
r4wsi2_second[4] <- sum(result$CorrectDummy)/nrow(result)

r4wsi2_pool <- (r4wsi2_first + r4wsi2_second)/2

# # results so far should be as follows:
# r4wsi2_first <- c(0.356, 0.878, 0.87, 0.932)
# r4wsi2_second <- c(0.318, 0.904, 0.94, 0.916)
# r4wsi2_pool <- c(0.337, 0.891, 0.905, 0.924)

### generate Figure 3
# define transparent colors for the figure
library("ggplot2")
forestgreen_t <- alpha("forestgreen", 0.35)
deepskyblue4_t <- alpha("deepskyblue4", 0.35)
darkorange2_t <- alpha("darkorange2", 0.35)
firebrick3_t <- alpha("firebrick3", 0.35)
mediumpurple3_t <- alpha("mediumpurple3", 0.35)
gold2_t <- alpha("gold2", 0.35)
aquamarine4_t <- alpha("aquamarine4", 0.35)
hotpink2_t <- alpha("hotpink2", 0.35)
firebrick4_t <- alpha("firebrick4", 0.35)
forestgreen <- "forestgreen"
deepskyblue4 <- "deepskyblue4"
darkorange2 <- "darkorange2"
firebrick3 <- "firebrick3"
mediumpurple3 <- "mediumpurple3"
gold2 <- "gold2"
aquamarine4 <- "aquamarine4"
hotpink2 <- "hotpink2"
firebrick4 <- "firebrick4"

# output plot in pdf
pdf("topicplots/figureSI7.pdf", width = 10, height = 7)
par(mgp = c(1.5, 0, 0), mar = c(2, 3, .7, .7))
plot(NULL,
     main = NA,
     ylim=c(0, 1.02), 
     xlim = c(0.6, 5), 
     ylab = "Proportion Correct",
     xlab = NA,
     cex.lab = 1.2,
     axes = F)
axis(side = 2, at = seq(0, 1, by = 0.2), col.ticks = NA, cex.axis = 1.2)
axis(side = 1, at = 1,
     labels = c('R4WSI\nRandom Documents'),
     las = 1,
     col = NA,
     col.ticks = NA,
     cex.axis = 1.2)
legend(1.4, 1.06,
       c("Model 1: 10-topic, one iteration",
         "Model 2: 10-topic",
         "Model 3: 50-topic",
         "Model 4: 100-topic",
         "Model 5: 500-topic"),
       col = c(forestgreen, deepskyblue4, darkorange2, firebrick3, mediumpurple3),
       lty = 1,
       lwd = 3,
       cex = 1.2,
       bty = 'n')
# ----------------------------------------------------------------
abline(h = 0.25, col = "gray", lty = 1, lwd = 2)
# -------------------------------- R4WSI with documents ---------------------------------
# 10k1it
points(x = 0.7, y = r4wsi_first[1], pch = 20, col = forestgreen_t, cex = 2)
segments(x0 = 0.7, y0 = r4wsi_first[1]-1.96*sqrt(r4wsi_first[1]*(1-r4wsi_first[1])/490), x1 = 0.7, y1 = r4wsi_first[1]+1.96*sqrt(r4wsi_first[1]*(1-r4wsi_first[1])/490), col = forestgreen_t, lwd = 3)
points(x = 0.72, y = r4wsi_second[1], pch = 20, col = forestgreen_t, cex = 2)
segments(x0 = 0.72, y0 = r4wsi_second[1]-1.96*sqrt(r4wsi_second[1]*(1-r4wsi_second[1])/500), x1 = 0.72, y1 = r4wsi_second[1]+1.96*sqrt(r4wsi_second[1]*(1-r4wsi_second[1])/500), col = forestgreen_t, lwd = 3)
points(x = 0.75, y = r4wsi_pool[1], pch = 20, col = forestgreen, cex = 2)
segments(x0 = 0.75, y0 = r4wsi_pool[1]-1.96*sqrt(r4wsi_pool[1]*(1-r4wsi_pool[1])/990), x1 = 0.75, y1 = r4wsi_pool[1]+1.96*sqrt(r4wsi_pool[1]*(1-r4wsi_pool[1])/990), col = forestgreen, lwd = 3)
# 10k  
points(x = 0.85, y = r4wsi_first[2], pch = 20, col = deepskyblue4_t, cex = 2)
segments(x0 = 0.85, y0 = r4wsi_first[2]-1.96*sqrt(r4wsi_first[2]*(1-r4wsi_first[2])/500), x1 = 0.85, y1 = r4wsi_first[2]+1.96*sqrt(r4wsi_first[2]*(1-r4wsi_first[2])/500), col = deepskyblue4_t, lwd = 3)
points(x = 0.87, y = r4wsi_second[2], pch = 20, col = deepskyblue4_t, cex = 2)
segments(x0 = 0.87, y0 = r4wsi_second[2]-1.96*sqrt(r4wsi_second[2]*(1-r4wsi_second[2])/500), x1 = 0.87, y1 = r4wsi_second[2]+1.96*sqrt(r4wsi_second[2]*(1-r4wsi_second[2])/500), col = deepskyblue4_t, lwd = 3)
points(x = 0.9, y = r4wsi_pool[2], pch = 20, col = deepskyblue4, cex = 2)
segments(x0 = 0.9, y0 = r4wsi_pool[2]-1.96*sqrt(r4wsi_pool[2]*(1-r4wsi_pool[2])/1000), x1 = 0.9, y1 = r4wsi_pool[2]+1.96*sqrt(r4wsi_pool[2]*(1-r4wsi_pool[2])/1000), col = deepskyblue4, lwd = 3)
# 50k
points(x = 1, y = r4wsi_first[3], pch = 20, col = darkorange2_t, cex = 2)
segments(x0 = 1, y0 = r4wsi_first[3]-1.96*sqrt(r4wsi_first[3]*(1-r4wsi_first[3])/500), x1 = 1, y1 = r4wsi_first[3]+1.96*sqrt(r4wsi_first[3]*(1-r4wsi_first[3])/500), col = darkorange2_t, lwd = 3)
points(x = 1.02, y = r4wsi_second[3], pch = 20, col = darkorange2_t, cex = 2)
segments(x0 = 1.02, y0 = r4wsi_second[3]-1.96*sqrt(r4wsi_second[3]*(1-r4wsi_second[3])/500), x1 = 1.02, y1 = r4wsi_second[3]+1.96*sqrt(r4wsi_second[3]*(1-r4wsi_second[3])/500), col = darkorange2_t, lwd = 3)
points(x = 1.05, y = r4wsi_pool[3], pch = 20, col = darkorange2, cex = 2)
segments(x0 = 1.05, y0 = r4wsi_pool[3]-1.96*sqrt(r4wsi_pool[3]*(1-r4wsi_pool[3])/1000), x1 = 1.05, y1 = r4wsi_pool[3]+1.96*sqrt(r4wsi_pool[3]*(1-r4wsi_pool[3])/1000), col = darkorange2, lwd = 3)
# 100k
points(x = 1.15, y = r4wsi_first[4], pch = 20, col = firebrick3_t, cex = 2)
segments(x0 = 1.15, y0 = r4wsi_first[4]-1.96*sqrt(r4wsi_first[4]*(1-r4wsi_first[4])/500), x1 = 1.15, y1 = r4wsi_first[4]+1.96*sqrt(r4wsi_first[4]*(1-r4wsi_first[4])/500), col = firebrick3_t, lwd = 3)
points(x = 1.17, y = r4wsi_second[4], pch = 20, col = firebrick3_t, cex = 2)
segments(x0 = 1.17, y0 = r4wsi_second[4]-1.96*sqrt(r4wsi_second[4]*(1-r4wsi_second[4])/500), x1 = 1.17, y1 = r4wsi_second[4]+1.96*sqrt(r4wsi_second[4]*(1-r4wsi_second[4])/500), col = firebrick3_t, lwd = 3)
points(x = 1.2, y = r4wsi_pool[4], pch = 20, col = firebrick3, cex = 2)
segments(x0 = 1.2, y0 = r4wsi_pool[4]-1.96*sqrt(r4wsi_pool[4]*(1-r4wsi_pool[4])/1000), x1 = 1.2, y1 = r4wsi_pool[4]+1.96*sqrt(r4wsi_pool[4]*(1-r4wsi_pool[4])/1000), col = firebrick3, lwd = 3)
# 500k 
points(x = 1.3, y = r4wsi_first[5], pch = 20, col = mediumpurple3_t, cex = 2)
segments(x0 = 1.3, y0 = r4wsi_first[5]-1.96*sqrt(r4wsi_first[5]*(1-r4wsi_first[5])/500), x1 = 1.3, y1 = r4wsi_first[5]+1.96*sqrt(r4wsi_first[5]*(1-r4wsi_first[5])/500), col = mediumpurple3_t, lwd = 3)
points(x = 1.32, y = r4wsi_second[5], pch = 20, col = mediumpurple3_t, cex = 2)
segments(x0 = 1.32, y0 = r4wsi_second[5]-1.96*sqrt(r4wsi_second[5]*(1-r4wsi_second[5])/500), x1 = 1.32, y1 = r4wsi_second[5]+1.96*sqrt(r4wsi_second[5]*(1-r4wsi_second[5])/500), col = mediumpurple3_t, lwd = 3)
points(x = 1.35, y = r4wsi_pool[5], pch = 20, col = mediumpurple3, cex = 2)
segments(x0 = 1.35, y0 = r4wsi_pool[5]-1.96*sqrt(r4wsi_pool[5]*(1-r4wsi_pool[5])/1000), x1 = 1.35, y1 = r4wsi_pool[5]+1.96*sqrt(r4wsi_pool[5]*(1-r4wsi_pool[5])/1000), col = mediumpurple3, lwd = 3)
# ------------------------------- p-values -------------------------------
segments(x0 = 0.75, y0 = 0.15, x1 = 0.75, y1 = 0.235, col = "gray", lty = 3)
segments(x0 = 0.75, y0 = 0.15, x1 = 0.9, y1 = 0.15, col = "gray", lty = 3)
segments(x0 = 0.9, y0 = 0.15, x1 = 0.9, y1 = 0.715, col = "gray", lty = 3)
text(x = 0.83, y = 0.12, "0.000")
segments(x0 = 1.05, y0 = 1, x1 = 1.05, y1 = 0.83, col = "gray", lty = 3)
segments(x0 = 1.05, y0 = 1, x1 = 1.2, y1 = 1, col = "gray", lty = 3)
segments(x0 = 1.2, y0 = 1, x1 = 1.2, y1 = 0.895, col = "gray", lty = 3)
text(x = 1.13, y = 1.03, "0.000")
segments(x0 = 1.2, y0 = 0.43, x1 = 1.2, y1 = 0.82, col = "gray", lty = 3)
segments(x0 = 1.2, y0 = 0.43, x1 = 1.35, y1 = 0.34, col = "gray", lty = 3)
segments(x0 = 1.35, y0 = 0.43, x1 = 1.35, y1 = 0.56, col = "gray", lty = 3)
text(x = 1.28, y = 0.40, "0.000")
# ---------------------------------------------------------------
axis(side = 1, at = 3,
     labels = c('R4WSI\nRepresentative Documents'),
     las = 1,
     col = NA,
     col.ticks = NA,
     cex.axis = 1.2)
legend(3.4, 0.78,
       c("Model 6: 20-topic, one iteration",
         "Model 7: 10-topic",
         "Model 8: 20-topic, covariates",
         "Model 9: 100-topic"),
       col = c(gold2, aquamarine4, hotpink2, firebrick4),
       lty = 1,
       lwd = 3,
       cex = 1.2,
       bty = 'n')
# -----------------------------------------------------------------
# 20k1it
points(x = 2.85, y = r4wsi2_first[1], pch = 20, col = gold2_t, cex = 2)
segments(x0 = 2.85, y0 = r4wsi2_first[1]-1.96*sqrt(r4wsi2_first[1]*(1-r4wsi2_first[1])/500), x1 = 2.85, y1 = r4wsi2_first[1]+1.96*sqrt(r4wsi2_first[1]*(1-r4wsi2_first[1])/500), col = gold2_t, lwd = 3)
points(x = 2.87, y = r4wsi2_second[1], pch = 20, col = gold2_t, cex = 2)
segments(x0 = 2.87, y0 = r4wsi2_second[1]-1.96*sqrt(r4wsi2_second[1]*(1-r4wsi2_second[1])/500), x1 = 2.87, y1 = r4wsi2_second[1]+1.96*sqrt(r4wsi2_second[1]*(1-r4wsi2_second[1])/500), col = gold2_t, lwd = 3)
points(x = 2.9, y = r4wsi2_pool[1], pch = 20, col = gold2, cex = 2)
segments(x0 = 2.9, y0 = r4wsi2_pool[1]-1.96*sqrt(r4wsi2_pool[1]*(1-r4wsi2_pool[1])/1000), x1 = 2.9, y1 = r4wsi2_pool[1]+1.96*sqrt(r4wsi2_pool[1]*(1-r4wsi2_pool[1])/1000), col = gold2, lwd = 3)
# 10k
points(x = 3, y = r4wsi2_first[2], pch = 20, col = aquamarine4_t, cex = 2)
segments(x0 = 3, y0 = r4wsi2_first[2]-1.96*sqrt(r4wsi2_first[2]*(1-r4wsi2_first[2])/500), x1 = 3, y1 = r4wsi2_first[2]+1.96*sqrt(r4wsi2_first[2]*(1-r4wsi2_first[2])/500), col = aquamarine4_t, lwd = 3)
points(x = 3.02, y = r4wsi2_second[2], pch = 20, col = aquamarine4_t, cex = 2)
segments(x0 = 3.02, y0 = r4wsi2_second[2]-1.96*sqrt(r4wsi2_second[2]*(1-r4wsi2_second[2])/500), x1 = 3.02, y1 = r4wsi2_second[2]+1.96*sqrt(r4wsi2_second[2]*(1-r4wsi2_second[2])/500), col = aquamarine4_t, lwd = 3)
points(x = 3.05, y = r4wsi2_pool[2], pch = 20, col = aquamarine4, cex = 2)
segments(x0 = 3.05, y0 = r4wsi2_pool[2]-1.96*sqrt(r4wsi2_pool[2]*(1-r4wsi2_pool[2])/1000), x1 = 3.05, y1 = r4wsi2_pool[2]+1.96*sqrt(r4wsi2_pool[2]*(1-r4wsi2_pool[2])/1000), col = aquamarine4, lwd = 3)
# 20k
points(x = 3.15, y = r4wsi2_first[3], pch = 20, col = hotpink2_t, cex = 2)
segments(x0 = 3.15, y0 = r4wsi2_first[3]-1.96*sqrt(r4wsi2_first[3]*(1-r4wsi2_first[3])/500), x1 = 3.15, y1 = r4wsi2_first[3]+1.96*sqrt(r4wsi2_first[3]*(1-r4wsi2_first[3])/500), col = hotpink2_t, lwd = 3)
points(x = 3.17, y = r4wsi2_second[3], pch = 20, col = hotpink2_t, cex = 2)
segments(x0 = 3.17, y0 = r4wsi2_second[3]-1.96*sqrt(r4wsi2_second[3]*(1-r4wsi2_second[3])/500), x1 = 3.17, y1 = r4wsi2_second[3]+1.96*sqrt(r4wsi2_second[3]*(1-r4wsi2_second[3])/500), col = hotpink2_t, lwd = 3)
points(x = 3.2, y = r4wsi2_pool[3], pch = 20, col = hotpink2, cex = 2)
segments(x0 = 3.2, y0 = r4wsi2_pool[3]-1.96*sqrt(r4wsi2_pool[3]*(1-r4wsi2_pool[3])/1000), x1 = 3.2, y1 = r4wsi2_pool[3]+1.96*sqrt(r4wsi2_pool[3]*(1-r4wsi2_pool[3])/1000), col = hotpink2, lwd = 3)
# 100k 
points(x = 3.3, y = r4wsi2_first[4], pch = 20, col = firebrick4_t, cex = 2)
segments(x0 = 3.3, y0 = r4wsi2_first[4]-1.96*sqrt(r4wsi2_first[4]*(1-r4wsi2_first[4])/500), x1 = 3.3, y1 = r4wsi2_first[4]+1.96*sqrt(r4wsi2_first[4]*(1-r4wsi2_first[4])/500), col = firebrick4_t, lwd = 3)
points(x = 3.32, y = r4wsi2_second[4], pch = 20, col = firebrick4_t, cex = 2)
segments(x0 = 3.32, y0 = r4wsi2_second[4]-1.96*sqrt(r4wsi2_second[4]*(1-r4wsi2_second[4])/500), x1 = 3.32, y1 = r4wsi2_second[4]+1.96*sqrt(r4wsi2_second[4]*(1-r4wsi2_second[4])/500), col = firebrick4_t, lwd = 3)
points(x = 3.35, y = r4wsi2_pool[4], pch = 20, col = firebrick4, cex = 2)
segments(x0 = 3.35, y0 = r4wsi2_pool[4]-1.96*sqrt(r4wsi2_pool[4]*(1-r4wsi2_pool[4])/1000), x1 = 3.35, y1 = r4wsi2_pool[4]+1.96*sqrt(r4wsi2_pool[4]*(1-r4wsi2_pool[4])/1000), col = firebrick4, lwd = 3)
# ------------------------------- p-values -------------------------------
segments(x0 = 2.9, y0 = 0.21, x1 = 2.9, y1 = 0.3, col = "gray", lty = 3)
segments(x0 = 2.9, y0 = 0.21, x1 = 3.05, y1 = 0.21, col = "gray", lty = 3)
segments(x0 = 3.05, y0 = 0.21, x1 = 3.05, y1 = 0.8, col = "gray", lty = 3)
text(x = 2.98, y = 0.18, "0.000")
segments(x0 = 3.35, y0 = 0.8, x1 = 3.35, y1 = 0.88, col = "gray", lty = 3)
segments(x0 = 3.05, y0 = 0.8, x1 = 3.35, y1 = 0.8, col = "gray", lty = 3)
segments(x0 = 3.05, y0 = 0.8, x1 = 3.05, y1 = 0.855, col = "gray", lty = 3)
text(x = 3.22, y = 0.77, "0.014")
dev.off()

### replicate Table SI6 (Workers' Agreement Rate in Topic Validation Task)
agreement <- matrix(NA, nrow = 3, ncol = 5)

load("../1_topicvalidation/topicrecords/WIstm10k1it.Rdata")
results1 <- read.csv("../1_topicvalidation/topicresults/WIstm10k1itresults1.csv")
results2 <- read.csv("../1_topicvalidation/topicresults/WIstm10k1itresults2.csv")
agreement[1, 1] <- as.numeric(checkAgree(results1, results2, key = record, type = "WI")[2])
load("../1_topicvalidation/topicrecords/WIstm10k.Rdata")
results1 <- read.csv("../1_topicvalidation/topicresults/WIstm10kresults1.csv")
results2 <- read.csv("../1_topicvalidation/topicresults/WIstm10kresults2.csv")
agreement[1, 2] <- as.numeric(checkAgree(results1, results2, key = record, type = "WI")[2])
load("../1_topicvalidation/topicrecords/WIstm50k.Rdata")
results1 <- read.csv("../1_topicvalidation/topicresults/WIstm50kresults1.csv")
results2 <- read.csv("../1_topicvalidation/topicresults/WIstm50kresults2.csv")
agreement[1, 3] <- as.numeric(checkAgree(results1, results2, key = record, type = "WI")[2])
load("../1_topicvalidation/topicrecords/WIstm100k.Rdata")
results1 <- read.csv("../1_topicvalidation/topicresults/WIstm100kresults1.csv")
results2 <- read.csv("../1_topicvalidation/topicresults/WIstm100kresults2.csv")
agreement[1, 4] <- as.numeric(checkAgree(results1, results2, key = record, type = "WI")[2])
load("../1_topicvalidation/topicrecords/WIstm500k.Rdata")
results1 <- read.csv("../1_topicvalidation/topicresults/WIstm500kresults1.csv")
results2 <- read.csv("../1_topicvalidation/topicresults/WIstm500kresults2.csv")
agreement[1, 5] <- as.numeric(checkAgree(results1, results2, key = record, type = "WI")[2])

load("../1_topicvalidation/topicrecords/T8WSIstm10k1it.Rdata")
results1 <- read.csv("../1_topicvalidation/topicresults/T8WSIstm10k1itresults1.csv")
results2 <- read.csv("../1_topicvalidation/topicresults/T8WSIstm10k1itresults2.csv")
agreement[2, 1] <- as.numeric(checkAgree(results1, results2, key = record, type = "T8WSI")[2])
load("../1_topicvalidation/topicrecords/T8WSIstm10k.Rdata")
results1 <- read.csv("../1_topicvalidation/topicresults/T8WSIstm10kresults1.csv")
results2 <- read.csv("../1_topicvalidation/topicresults/T8WSIstm10kresults2.csv")
agreement[2, 2] <- as.numeric(checkAgree(results1, results2, key = record, type = "T8WSI")[2])
load("../1_topicvalidation/topicrecords/T8WSIstm50k.Rdata")
results1 <- read.csv("../1_topicvalidation/topicresults/T8WSIstm50kresults1.csv")
results2 <- read.csv("../1_topicvalidation/topicresults/T8WSIstm50kresults2.csv")
agreement[2, 3] <- as.numeric(checkAgree(results1, results2, key = record, type = "T8WSI")[2])
load("../1_topicvalidation/topicrecords/T8WSIstm100k.Rdata")
results1 <- read.csv("../1_topicvalidation/topicresults/T8WSIstm100kresults1.csv")
results2 <- read.csv("../1_topicvalidation/topicresults/T8WSIstm100kresults2.csv")
agreement[2, 4] <- as.numeric(checkAgree(results1, results2, key = record, type = "T8WSI")[2])
load("../1_topicvalidation/topicrecords/T8WSIstm500k.Rdata")
results1 <- read.csv("../1_topicvalidation/topicresults/T8WSIstm500kresults1.csv")
results2 <- read.csv("../1_topicvalidation/topicresults/T8WSIstm500kresults2.csv")
agreement[2, 5] <- as.numeric(checkAgree(results1, results2, key = record, type = "T8WSI")[2])

load("../1_topicvalidation/topicrecords/R4WSIstm10k1it.Rdata")
results1 <- read.csv("../1_topicvalidation/topicresults/R4WSIstm10k1itresults1.csv")
results2 <- read.csv("../1_topicvalidation/topicresults/R4WSIstm10k1itresults2.csv")
agreement[3, 1] <- as.numeric(checkAgree(results1, results2, key = record, type = "R4WSI")[2])
load("../1_topicvalidation/topicrecords/R4WSIstm10k.Rdata")
results1 <- read.csv("../1_topicvalidation/topicresults/R4WSIstm10kresults1.csv")
results2 <- read.csv("../1_topicvalidation/topicresults/R4WSIstm10kresults2.csv")
agreement[3, 2] <- as.numeric(checkAgree(results1, results2, key = record, type = "R4WSI")[2])
load("../1_topicvalidation/topicrecords/R4WSIstm50k.Rdata")
results1 <- read.csv("../1_topicvalidation/topicresults/R4WSIstm50kresults1.csv")
results2 <- read.csv("../1_topicvalidation/topicresults/R4WSIstm50kresults2.csv")
agreement[3, 3] <- as.numeric(checkAgree(results1, results2, key = record, type = "R4WSI")[2])
load("../1_topicvalidation/topicrecords/R4WSIstm100k.Rdata")
results1 <- read.csv("../1_topicvalidation/topicresults/R4WSIstm100kresults1.csv")
results2 <- read.csv("../1_topicvalidation/topicresults/R4WSIstm100kresults2.csv")
agreement[3, 4] <- as.numeric(checkAgree(results1, results2, key = record, type = "R4WSI")[2])
load("../1_topicvalidation/topicrecords/R4WSIstm500k.Rdata")
results1 <- read.csv("../1_topicvalidation/topicresults/R4WSIstm500kresults1.csv")
results2 <- read.csv("../1_topicvalidation/topicresults/R4WSIstm500kresults2.csv")
agreement[3, 5] <- as.numeric(checkAgree(results1, results2, key = record, type = "R4WSI")[2])

# table SI6
print(agreement)